import scrapy
import json
import jsonpath
import re
from kuaishou.items import KuaishouItem

class Douyin(scrapy.Spider):
    name = "douyin"
    allowed_domains = []  # 爬虫有效域
    start_urls = ['https://aweme.snssdk.com/aweme/v1/feed/?type=0&max_cursor=0&min_cursor=0&count=6&volume=0.0&pull_type=0&ts=1532512822&app_type=normal&openudid=58946b2d53d47451&version_name=1.7.6&device_type=SM-G955N&ssmix=a&iid=38960415174&os_api=19&device_id=55329872423&resolution=720*1280&device_brand=samsung&aid=1128&manifest_version_code=176&app_name=aweme&_rticket=1532512823822&os_version=4.4.2&device_platform=android&version_code=176&update_version_code=1762&ac=wifi&dpi=240&uuid=354730010589418&language=zh&channel=wap_aweme_homepage&as=a125246586e33bfaa86633&cp=463abe5261805daee1dmcm&mas=00966a8af6dbc6708882bdb2ddf3db4930cccc6c6c1c86866646cc']
    custom_settings = {
        'DEFAULT_REQUEST_HEADERS' : {
            "Host": "aweme.snssdk.com",
            "Connection": "Keep-Alive",
            "Accept-Encoding": "gzip",
            "Cookie": "install_id=38960415174; ttreq=1$3d18f903662155a730d6c8f4f568bfe551985723; qh[360]=1; odin_tt=7a41413736494f4364794236727270471f7929c14aed7c18b7cb3b12371a6993eed35c81ad6bfc9fc8acf02c8228101f",
            "User-Agent": "okhttp/3.8.1",
        },

        "ITEM_PIPELINES": {
            'kuaishou.pipelines.KuaishouPipeline': 300,
        },

    }

    def parse(self, response):
        item = KuaishouItem()
        print(response.text)
        data_json = jsonpath.jsonpath(json.loads(response.text),'$..aweme_list')
        for data in data_json:
            for d in data:
                user_id = d["author"]["short_id"]
                user_name = d["author"]["nickname"]
                mv_url = d["video"]["play_addr_lowbr"]["url_list"][0]
                caption =self.split_n(d["desc"])
                music_name = d["music"]["title"]
                comment_count = d["statistics"]["comment_count"]
                like_count = d["statistics"]["digg_count"]
                view_count = d["statistics"]["digg_count"]

                item["user_id"] = user_id
                item["user_name"] = user_name
                item["mv_url"] = mv_url
                item["caption"] = caption
                item["music_name"] = music_name
                item["comment_count"] = comment_count
                item["like_count"] = like_count
                item["view_count"] = view_count
                yield item

    # 去除/n符号 表情符号
    def split_n(self, ags):
        try:
            # python UCS-4 build的处理方式
            highpoints = re.compile(u'[\U00010000-\U0010ffff]')
        except re.error:
            # python UCS-2 build的处理方式
            highpoints = re.compile(u'[\uD800-\uDBFF][\uDC00-\uDFFF]')

        resovle_value = highpoints.sub(u'??', ags)

        return re.sub('\n', '-', resovle_value)
